import datetime
import json
import re
import sqlite3
import time
import traceback

import requests
from requests_html import HTMLSession
from tenacity import retry

from conf import setting

sep = "    "
token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIyLVp1c2xDSWptOWQ3TGZjd3VMcmp1TGlSVVU4d1U3aFwvcG1OUzI5a2tYOFo2NnBZS3RNK0hBQ2dMN1R0M3pZU1p1RUJrazJJMDFMZ2tzK2c4S25EQXdJcGZiZU8zK20wZVg4NzY0OE9JMnBzVEpYQT09IiwidWlkIjoiNjA3ZWE0YTg0Yzc3NWUwNDk3ZjVjMWQ1Iiwic3JjIjoiYW1pbmVyIiwicm9sZXMiOltdLCJpc3MiOiJhcGkuYW1pbmVyLm9yZyIsImV4cCI6MTYyMjI0NTgzNywiaWF0IjoxNjE5NjUzODM3LCJqdGkiOiIxZTU5ODI0NTdjODRhNmFmNjgyOTFmZmY5YTFiYWYyMTgxZmVkMzBjZTQyNjFlNTEzYTMzZjlkMmQxOTE2YTk4YWY1ZTVkOWJmY2I3YzUyZTRjNDkyOTUzMGUyYTRlZmIwNDQ4ZWQ1NDFlNGUzMThkM2NmM2Q0Nzc0Mzg0OTlmYzljODc3YzY5NGM2NjA4MTU2MGRiMDExNzllNDIzOTQxOGRiMWNlNGY1M2M5ZmM5MTdjODY5NWUwN2FkMTc1ODg5Y2ZlODU1NjA2Mzg4ZTE0MzdkOGRmYmNlMDA1ZmE3NzAxN2VlMWEwZjhlNzFjNWQzNDRhYTdlNzZhOTJjOWY0IiwiZW1haWwiOiJwZWlzZXFpc3NsaDAwQDE2My5jb20ifQ.zHfI5Ppo9R8saACp4sLVHs6vsMGSkS70NTaSEfVRoX0"
headers = setting.HEADERS
headers["Authorization"] = token
session1 = requests.Session()
session1.cookies.set("Authorization", f'"{token}"')
session2 = HTMLSession()
session2.cookies.set("Authorization", f'"{token}"')


def database():
    client = sqlite3.connect("aminer.db")
    db = client.cursor()
    db.execute('''CREATE TABLE IF NOT EXISTS person (
		id integer PRIMARY KEY AUTOINCREMENT,
		person_id TEXT,
		url TEXT,
		"name" TEXT,
		avatar TEXT,
		"position" TEXT,
		affiliation TEXT,
		phone TEXT,
		fax TEXT,
		email TEXT,
		homepage TEXT,
		address TEXT,
		"work" TEXT,
		educational_background TEXT,
		profile TEXT,
		research_interest TEXT,
		"query" TEXT,
		create_datetime TEXT,
	flag integer DEFAULT 0 
	)''')
    db.execute('''CREATE TABLE IF NOT EXISTS paper (
    id integer PRIMARY KEY AUTOINCREMENT,
    person_id TEXT,
    paper_id TEXT,
    title TEXT,
    url TEXT,
    author TEXT,
    create_datetime TEXT,
    "data" TEXT
    )''')
    db.execute('''CREATE TABLE IF NOT EXISTS patent (
		id integer PRIMARY KEY AUTOINCREMENT,
		person_id TEXT,
		patent_id TEXT,
		title TEXT,
		app_date TEXT,
		applicant TEXT,
		"type" TEXT,
		create_datetime TEXT,
	"data" TEXT 
	);''')
    db.execute('''CREATE TABLE IF NOT EXISTS research_project (
		id integer PRIMARY KEY AUTOINCREMENT,
		person_id TEXT,
		research_project_id TEXT,
		title TEXT,
		start_date TEXT,
		end_date TEXT,
		register_number TEXT,
		amount TEXT,
		create_datetime TEXT,
	"data" TEXT 
	);''')
    return client, db


client, db = database()


def person_list(query: str, offset: int):
    print("person_list", query, offset)
    time.sleep(3)
    param = [{"action": "searchapi.SearchPerson", "parameters": {"offset": offset, "size": 20, "query": query, "include": ["agg", "intelli", "topics"]}, "schema": {"person": ["id", "name", "name_zh", "avatar", "tags", "is_follow", "num_view", "num_follow", "is_upvoted", "num_upvoted", "is_downvoted", "bind", {"profile": ["position", "position_zh", "affiliation", "affiliation_zh", "org"]}, {"indices": ["hindex", "gindex", "pubs", "citations", "newStar", "risingStar", "activity", "diversity", "sociability"]}, "tags_translated_zh"]}}]
    response = session1.post(url="https://apiv2.aminer.cn/n?a=__searchapi.SearchPerson___", data=json.dumps(param), headers=headers, timeout=60)
    if response.status_code == 200:
        response_data = response.json()
        total = int(response_data.get("data")[0].get("pager").get("total"))
        for item in response_data.get("data")[0].get("persons"):
            person_id = item.get("id")
            name = item.get("name")
            result = re.findall(re.compile("[a-zA-Z]+"), name)
            name = '-'.join(result).lower()
            db.execute("SELECT person_id FROM person WHERE flag=1 AND person_id=?", (person_id,))
            if not db.fetchone():
                get_all(person_id, name, query)
                db.execute("UPDATE person SET flag=1 WHERE person_id=?", (person_id,))
                client.commit()
        offset = offset + 20
        if offset < total:
            person_list(query, offset)


@retry(reraise=True)
def person_detail(person_id: str, name: str, query: str):
    print("person_detail", person_id, name, query)
    try:
        time.sleep(3)
        url = f"https://www.aminer.cn/profile/{name}/{person_id}"
        response = session2.get(url=url, headers=headers, timeout=60)
        if response.status_code == 200:
            # 名称
            name = response.html.xpath('//*[@class="info_zone"]//*[@class="name"]', first=True).text
            # 头像
            avatar = response.html.xpath('//*[@class="avatar_certified"]/img/@src', first=True)
            # 职称
            position = sep.join(response.html.xpath('//div[@class="expert_info_content"]//use[@*="#icon-position"]/parent::*/following-sibling::*[1]/text()'))
            # 专业
            affiliation = sep.join(response.html.xpath('//div[@class="expert_info_content"]//use[@*="#icon-affiliation"]/parent::*/following-sibling::*[1]/text()'))
            # 电话
            phone = sep.join(response.html.xpath('//div[@class="expert_info_content"]//use[@*="#icon-phone"]/parent::*/following-sibling::*[1]/text()'))
            # 传真
            fax = sep.join(response.html.xpath('//div[@class="expert_info_content"]//use[@*="#icon-fax"]/parent::*/following-sibling::*[1]/text()'))
            # 邮箱
            email = sep.join(response.html.xpath('//div[@class="expert_info_content"]//use[@*="#icon-email"]/parent::*/following-sibling::*[1]/text()'))
            # 个人主页
            homepage = sep.join(response.html.xpath('//div[@class="expert_info_content"]//use[@*="#icon-global"]/parent::*/following-sibling::*[1]/text()'))
            # 地址
            address = sep.join(response.html.xpath('//div[@class="expert_info_content"]//use[@*="#icon-address"]/parent::*/following-sibling::*[1]/text()'))
            result = re.search(re.compile('window.g_initialProps = .*;'), response.text)
            # 工作
            work = None
            if result:
                json_data = json.loads(result.group().replace("window.g_initialProps = ", "")[:-1])
                profile3 = json_data.get("profile").get("profile").get("profile")
                if profile3:
                    work = profile3.get("work")
            # 教育背景
            educational_background = response.html.xpath('//*[@class="a-aminer-core-profile-c-resume-card-resumeCard"]/div[@class="card_content"]')[1].text
            # 个人简介
            profile = response.html.xpath('//*[@class="a-aminer-core-profile-c-resume-card-resumeCard"]/div[@class="card_content"]')[2].text
            db.execute("SELECT person_id, query FROM person WHERE person_id=?", (person_id,))
            row = db.fetchone()
            if not row:
                db.execute("INSERT INTO person (person_id, url, `name`, avatar, `position`, affiliation, phone, fax, email, homepage, address, `work`, educational_background, profile, query, create_datetime) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (person_id, url, name, avatar, position, affiliation, phone, fax, email, homepage, address, work, educational_background, profile, query, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
                client.commit()
                print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), person_id, name, avatar, position, affiliation, phone, fax, email, homepage, address, work, educational_background, profile, query)
            else:
                db.execute("UPDATE person SET query=? WHERE id=?", (row[1] + sep + query, person_id))
                client.commit()
    except Exception as e:
        traceback.print_exc()
        raise e


# 研究兴趣
def get_research_interest(person_id: str):
    print("research_interest", person_id)
    time.sleep(3)
    param = [{"action": "PersonInterest.Get", "parameters": {"id": person_id, "is_year": True}}]
    response = session1.post(url="https://apiv2.aminer.cn/n?a=__PersonInterest.Get___", data=json.dumps(param), headers=headers, timeout=60)
    if response.status_code == 200:
        response_data = response.json()
        if response_data.get("data")[0].get("data").get("data"):
            for item in response_data.get("data")[0].get("data").get("data"):
                # 研究兴趣
                research_interest = item.get("t")
                db.execute("UPDATE person SET research_interest=? WHERE person_id=?", (research_interest, person_id))
                client.commit()


# 论文
def get_paper(person_id: str, offset: int):
    print("get_paper", person_id, offset)
    time.sleep(3)
    param = [{"action": "person.GetPersonPubs", "parameters": {"offset": offset, "size": 20, "sorts": ["!year"], "ids": [person_id], "searchType": "all"}, "schema": {"publication": ["id", "year", "title", "title_zh", "authors._id", "authors.name", "authors.name_zh", "num_citation", "venue.info.name", "venue.volume", "venue.info.name_zh", "venue.issue", "pages.start", "pages.end", "lang", "pdf", "doi", "urls", "versions"]}}]
    response = session1.post(url="https://apiv2.aminer.cn/magic?a=GetPersonPubs__person.GetPersonPubs___", data=json.dumps(param), headers=headers, timeout=60)
    if response.status_code == 200:
        response_data = response.json()
        if response_data.get("data")[0].get("keyValues"):
            total = int(response_data.get("data")[0].get("keyValues").get("total"))
            if response_data.get("data")[0].get("items"):
                for item in response_data.get("data")[0].get("items"):
                    id = item.get("id")
                    # 标题
                    title = item.get("title")
                    # 链接
                    url = None
                    if item.get("urls"):
                        url = sep.join(item["urls"])
                    author_list = []
                    for auth in item.get("authors"):
                        if auth.get("name"):
                            author_list.append(auth.get("name"))
                    # 作者
                    author = sep.join(author_list)
                    db.execute("SELECT paper_id FROM paper WHERE paper_id=?", (id,))
                    if not db.fetchone():
                        db.execute("INSERT INTO paper (paper_id, person_id, title, url, author, `data`, create_datetime) values (?, ?, ?, ?, ?, ?, ?)", (id, person_id, title, url, author, json.dumps(item), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
                        client.commit()
                        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), id, person_id, title, url, author)
            offset = offset + 20
            if offset < total:
                get_paper(person_id, offset)


# 专利
def get_patent(person_id: str, offset: int):
    print("get_patent", person_id, offset)
    time.sleep(3)
    response = session1.get(url=f"https://api.aminer.cn/api/person/patents/{person_id}/all/year/{offset}/20", headers=headers, timeout=60)
    if response.status_code == 200:
        response_data = response.json()
        total = int(response_data.get("size"))
        if response_data.get("patents"):
            for item in response_data.get("patents"):
                id = item.get("id")
                # 标题
                title = item.get("title") or item.get("title_zh")
                # 申请日期
                app_date = item.get("app_date")
                applicant_list = []
                if item.get("applicants"):
                    for appli in item.get("applicants"):
                        applicant_list.append(appli.get("name"))
                # 申请(专利权)人
                applicant = sep.join(applicant_list)
                # 专利类型
                type = None
                if item.get("type") == 1:
                    type = "发明专利"
                db.execute("SELECT patent_id FROM patent WHERE patent_id=?", (id,))
                if not db.fetchone():
                    db.execute("INSERT INTO patent (patent_id, person_id, title, app_date, applicant, type, `data`, create_datetime) values (?, ?, ?, ?, ?, ?, ?, ?)", (id, person_id, title, app_date, applicant, type, json.dumps(item), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
                    client.commit()
                    print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), id, person_id, title, app_date, applicant)
        offset = offset + 20
        if offset < total:
            get_patent(person_id, offset)


# 科研项目
def get_research_project(person_id: str, offset: int):
    print("get_research_project", person_id, offset)
    time.sleep(3)
    param = [{"action": "person.GetFundsByPersonID", "parameters": {"id": person_id, "end": 20, "start": offset}}]
    response = session1.post(url="https://apiv2.aminer.cn/magic?a=GetFundsByPersonID__person.GetFundsByPersonID___", data=json.dumps(param), headers=headers, timeout=60)
    if response.status_code == 200:
        response_data = response.json()
        if response_data.get("data")[0].get("keyValues"):
            total = int(response_data.get("data")[0].get("keyValues").get("total"))
            if response_data.get("data")[0].get("items"):
                for item in response_data.get("data")[0].get("items"):
                    id = item.get("id")
                    # 标题
                    title = item.get("title")
                    # 立项时间
                    start_date = item.get("start_date")
                    # 立项时间
                    end_date = item.get("end_date")
                    # 项目编号
                    register_number = item.get("register_number")
                    # 项目经费
                    amount = item.get("amount")
                    db.execute("SELECT research_project_id FROM research_project WHERE research_project_id=?", (id,))
                    if not db.fetchone():
                        db.execute("INSERT INTO research_project (research_project_id, person_id, title, start_date, end_date, register_number, amount, `data`,create_datetime) values (?, ?, ?, ?, ?, ?, ?, ?, ?)", (id, person_id, title, start_date, end_date, register_number, amount, json.dumps(item), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
                        client.commit()
                        print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), id, person_id, title, start_date, end_date)
            offset = offset + 20
            if offset < total:
                get_research_project(person_id, offset)


def get_all(person_id: str, name: str, query: str):
    person_detail(person_id, name, query)
    get_research_interest(person_id)
    get_paper(person_id, 0)
    get_research_project(person_id, 0)
    get_patent(person_id, 0)


# https://www.aminer.cn/
def main():
    # query_list = ["集成电路", "芯片", "微纳电子"]
    query_list = ["芯片", "微纳电子"]
    for query in query_list:
        person_list(query, 0)


if __name__ == '__main__':
    main()
